//==============================================================================================================================================================
// Author(s): Roman Khmelichek Wei Jiang
//
// The Parser is used for parsing symbol HTML pages. Support both the Trec data format(gov2) and the WARC data format(clueweb2009).
//==============================================================================================================================================================

#ifndef PARSER_H_
#define PARSER_H_

#include <boost/algorithm/string.hpp>
#include <cassert>
#include <cctype>
#include <stdint.h>
#include <strings.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <map>
#include "config_file_properties.h"
#include "configuration.h"

using namespace std;
using namespace boost;
template<class Callback>
  class Parser {
  public:
    enum ParsingMode {
      kSingleDoc, kManyDoc
    };

    enum DocType {
      kStandard, kTrec, kWarc, kNoSuchDocType
    };

    enum Context {
      kContextP = 0, // Plain.
      kContextB = 1, // Bold.
      kContextH = 2, // Heading.
      kContextI = 4, // Italic.
      kContextT = 8, // Title.
      kContextU = 16 // URL.
    };

    enum Tag {
      kTagNot, kTagB, kTagI, kTagH, kTagTitle, kTagScript, kTagDoc, kTagDocno, kTagDochdr, kTagHtml
    };

    struct WarcHeader {
      int content_length;

      const char* url;
      int url_len;

      const char* docno;
      int docno_len;

      bool fastPrioritySkip;
    };

    Parser(const ParsingMode& parsing_mode, const DocType& doc_type, Callback* callback);

    int LoadClueweb2009SpamReportDict(ifstream &inputfileForDocument);

    void LoadUpGov2DocumentSetNeededToBeExtracted();

    int getHamProbablity(long offset, string WARCTRECIDNeededToLookUp);

    int ParseDocumentCollection(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length);

    int ParseDocumentCollectionAndExtractingInfoForPhase2Pruning(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length);

    int ShowContentAndSimpleParse(const char* buf, int buf_len);

    int ShowContentOfSpecifcDocument(const char* buf, int buf_len, const long beginningPosition, const long endingPosition,const char* term, int term_len);

    int ShowContentOfSpecifcDocument(const char* buf, int buf_len, const long beginningPosition, const long endingPosition,vector<string> &queryID_Term_docIDList, string currentDocID, ofstream &outputFileHandler);

    int ParseWarcDataBuffer(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p, WarcHeader* header);

    int ParseWarcDataBufferForWeiToExtractPositionsAndContext(const char* buf, int buf_len,const long beginningPosition, const long endingPosition);

    void ParseIncomingPostingStreamForFormat1();
    void ParseIncomingPostingStreamForFormat2();
    void ParseIncomingPostingStreamForFormat3();
    void ParseIncomingPostingStreamForFormat4();
    void BuildInvertedIndexForDocHits();
    void BuildInvertedIndexForDocHitsMethod2();
    void BuildInvertedIndexForPostingHits();
    void BuildInvertedIndexForPostingHitAndDocHit();
    void BuildInvertedIndexForNormalizedPostingHit();
    void BuildInvertedIndexForDocHitsAndPostingHitHybridMethod_20141212();
    void BuildInvertedIndexForDocHitsAndPostingHitHybridMethod_withGlobalUPPAdded_20141215();

    int ParseTrecDataBuffer(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p);

    // Updated by Wei on 2015/02/24
    int ParseTrecDataBufferForBigrams(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p);

    // Updated by Wei 2013/07/29 night, nearly NO USE at all. And please sue the next following function instead.
    // The currently used function is called: ParseTrecDataBufferForPhase2PruningNewTryUpdatedOn20130714Night
    int ParseTrecDataBufferForPhase2Pruning(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p);

    // Currently used since 2013/11/04 afternoon at school by Wei
    int ParseTrecDataBufferForSelectedParsingUpdatedOn20131104Afternoon(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p, bool usePriorityDocumentListFlag);

    // NOT used function for Parse Trec Data Buffer For Phase2 Pruning since 2013/11/04 afternoon
    // This function has been taken place by the function called: ParseTrecDataBufferForSelectedParsingUpdatedOn20131104Afternoon(...)
    int ParseTrecDataBufferForPhase2PruningNewTryUpdatedOn20130714Night(const char* buf, int buf_len, uint32_t& doc_id, int& avg_doc_length, const char*& curr_p, bool usePriorityDocumentListFlag);

    Tag ProcessTag(const char* tag, int tag_len, bool& in_closing_tag, uint32_t doc_id);

    bool IsValidTag(const char* curr_tag_p, const char* tag, int tag_len, const char curr_tag_name[]);

    bool IsWithinBounds(const char* curr, const char* start, int len);

    // Handles the case where you have a tag that starts with the same characters, but the ending characters could be anything within a range.
    bool AreValidTags(const char* curr_tag_p, const char* tag, int tag_len, const char curr_tag_base[], const char start_range[], const char end_range[],
                      int range_len);

    string parseURLForDomainName(string inputURL);

    // Processes the WARC Response Header. Returns the number of bytes read from the header.
    int ProcessWarcResponseAndHTTPHeader(const char* buf, int buf_len, const char* curr_p, WarcHeader* header, uint32_t& doc_id);

    // Processes the WARC Info Header. Returns the number of bytes read from the header.
    int ProcessWarcInfoHeader(const char* buf, int buf_len, const char* curr_p, WarcHeader* header);

    bool IsIndexable(char c) {
      return isalnum(static_cast<unsigned char> (c));
    }

    void BitSet(unsigned char& b1, unsigned char b2) {
      b1 |= b2;
    }

    void BitUnset(unsigned char& b1, unsigned char b2) {
      b1 &= ~b2;
    }

    bool BitCheck(unsigned char b1, unsigned char b2) {
      return b1 & b2;
    }

    void UpdateContext(unsigned char& context, bool closing_tag, unsigned char set_context) {
      if (!closing_tag)
        BitSet(context, set_context);
      else
        BitUnset(context, set_context);
    }

    static DocType GetDocumentCollectionFormat(const char* doc_type_str) {
      if (strcasecmp("trec", doc_type_str) == 0) {
        return kTrec;
      } else if (strcasecmp("warc", doc_type_str) == 0) {
        return kWarc;
      } else {
        return kNoSuchDocType;
      }
    }

  private:
    ParsingMode parsing_mode_;
    DocType doc_type_;
    Callback* callback_;
    map<uint32_t,int> priority_docIDs_;
  };

//because this code will have to be inline and very large, so place it at the end of this file.
#include "parser-inl.h"

#endif /* PARSER_H_ */
